﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.Serialization;
using System.ServiceModel;
using System.Text;

namespace BilingualSentIndexing
{
    // NOTE: If you change the class name "Service1" here, you must also update the reference to "Service1" in App.config.
    [ServiceBehavior(InstanceContextMode = InstanceContextMode.Single, ConcurrencyMode = ConcurrencyMode.Multiple)]
    [FaultErrorHandler]
    public class BilingualSentIndexingService : IBilingualSentIndexingService
    {
        private TermIndexReader language2TermIndexReader;
        private TermIndexReader language1TermIndexReader;
        private SentIdPairReader sentIdPairReader;
        private SentPairReader sentPairReader;

        private static int iRankLevelFactor = 100000;
        private ChineseUtility.Synonym chsSynonym;
        private ChineseUtility.WordBreaker chsWordBreaker;
        private JapaneseUtility.WordBreaker jpnWordBreaker;
        private DictReader dictReader;
        private object _lockObject = new object();

        public BilingualSentIndexingService()
        {
            string language2Term2OffsetFileName;
            string language2SentIdIndexFileName;
            string language1Term2OffsetFileName;
            string language1SentIdIndexFileName;
            string sentenceIdPairFileName;
            string sentencePairFileName;
            string dictFileName;

            lock (_lockObject)
            {
                string strServiceRootPath = System.Configuration.ConfigurationSettings.AppSettings["ServiceRootPath"];

                dictFileName = strServiceRootPath + @"\Dict.dat";
                dictReader = new DictReader();
                dictReader.Open(dictFileName);


                string chineseDictionaryFileName, synonymDictionaryFileName;
                string japaneseDictionaryFileName;

                chineseDictionaryFileName = strServiceRootPath + @"\ChineseDictionary.txt";
                synonymDictionaryFileName = strServiceRootPath + @"\SynonymDictionary.txt";
                japaneseDictionaryFileName = strServiceRootPath + @"\JapaneseDictionary.txt";

                chsWordBreaker = new ChineseUtility.WordBreaker(chineseDictionaryFileName);
                chsSynonym = new ChineseUtility.Synonym(synonymDictionaryFileName, chineseDictionaryFileName);

                if (System.IO.File.Exists(japaneseDictionaryFileName) == true)
                {
                    jpnWordBreaker = new JapaneseUtility.WordBreaker(japaneseDictionaryFileName);
                }
                else
                {
                    jpnWordBreaker = null;
                }

                language1Term2OffsetFileName = strServiceRootPath + @"\Language1Term2Offset.dat";
                language1SentIdIndexFileName = strServiceRootPath + @"\Language1SentIdIndex.dat";
                language2Term2OffsetFileName = strServiceRootPath + @"\Language2Term2Offset.dat";
                language2SentIdIndexFileName = strServiceRootPath + @"\Language2SentIdIndex.dat";
                sentenceIdPairFileName = strServiceRootPath + @"\SentIdPair.dat";
                sentencePairFileName = strServiceRootPath + @"\SentPair.dat";

                double fPercentTermIndexInMemory = double.Parse(System.Configuration.ConfigurationSettings.AppSettings["PercentTermIndexInMemory"].ToString());

                language1TermIndexReader = new TermIndexReader();
                language1TermIndexReader.Open(language1Term2OffsetFileName, language1SentIdIndexFileName, fPercentTermIndexInMemory);
                language2TermIndexReader = new TermIndexReader();
                language2TermIndexReader.Open(language2Term2OffsetFileName, language2SentIdIndexFileName, fPercentTermIndexInMemory);

                double fPercentSentPairIdInMemory = double.Parse(System.Configuration.ConfigurationSettings.AppSettings["PercentSentPairIdInMemory"].ToString());
                sentIdPairReader = new SentIdPairReader();
                sentIdPairReader.Open(sentenceIdPairFileName, fPercentSentPairIdInMemory);

                double fPercentCorpusInMemory = double.Parse(System.Configuration.ConfigurationSettings.AppSettings["PercentCorpusInMemory"].ToString());
                sentPairReader = new SentPairReader();
                sentPairReader.Open(sentencePairFileName, fPercentCorpusInMemory);
            }
        }

        private List<SentenceIdPair> GetSentencePairByIDList(ref List<int> sentenceIndexList)
        {
            List<SentenceIdPair> sentpairIdList = new List<SentenceIdPair>();
            foreach (int offset in sentenceIndexList)
            {
                SentenceIdPair sentIdPair = new SentenceIdPair();
                sentIdPair.score = 0;
                sentIdPairReader.Read(out sentIdPair.aiLanguage1Line, out sentIdPair.aiLanguage2Line, out sentIdPair.sentPairOffset, offset);

                sentpairIdList.Add(sentIdPair);
            }
            return sentpairIdList;
        }

        private Dictionary<int, int[]> QueryWord2SentencePairId(ref SortedList<int, QueryTermProp> queryIdList, ref TermIndexReader reader)
        {
            Dictionary<int, int[]> word2SentId = new Dictionary<int, int[]>();
            foreach(KeyValuePair<int, QueryTermProp> pair in queryIdList)
            {
                int[] aiIndex = reader.ReadTermSentenceIndex(pair.Key);
                if (aiIndex != null)
                {
                    int keyword = pair.Value.synSrcTerm;
                    if (word2SentId.ContainsKey(keyword) == false)
                    {
                        word2SentId.Add(keyword, aiIndex);
                    }
                    else
                    {
                        int[] aiNewIndex;
                        aiNewIndex = new int[aiIndex.Length + word2SentId[keyword].Length];
                        System.Buffer.BlockCopy(word2SentId[keyword], 0, aiNewIndex, 0, word2SentId[keyword].Length * sizeof(int));
                        System.Buffer.BlockCopy(aiIndex, 0, aiNewIndex, word2SentId[keyword].Length * sizeof(int), aiIndex.Length * sizeof(int));
                        word2SentId[keyword] = aiNewIndex;
                    }
                }
            }

            foreach (KeyValuePair<int, int[]> pair in word2SentId)
            {
                Array.Sort<int>(pair.Value);
            }

            return word2SentId;
        }

      


        private List<int> QueryCommonSentence(ref Dictionary<int, int[]> word2SentId)
        {
            int minCount = 9999999;
            int keyWord = -1;
            foreach (KeyValuePair<int, int[]> pair in word2SentId)
            {
                if (pair.Value.Length < minCount)
                {
                    keyWord = pair.Key;
                    minCount = pair.Value.Length;
                }
            }

            if (minCount < 9999999)
            {
                List<int> resultList = new List<int>(4000);
                int lastStandIndex = -1;
                foreach (int standIndex in word2SentId[keyWord])
                {
                    if (standIndex == lastStandIndex)
                    {
                        continue;
                    }

                    bool flag = true;
                    foreach (KeyValuePair<int, int[]> pair in word2SentId)
                    {
                        if (pair.Key == keyWord)
                        {
                            continue;
                        }

                        if (Array.BinarySearch<int>(pair.Value, standIndex) < 0)
                        {
                            flag = false;
                            break;
                        }
                    }

                    if (flag == true)
                    {
                        resultList.Add(standIndex);
                        if (resultList.Count > 4000)
                        {
                            return resultList;
                        }
                    }

                    lastStandIndex = standIndex;
                }
                return resultList;
            }

            return null;
        }

        #region IBilingualSentIndexingService Members


        private List<SentencePair> RankSentencePair(ref List<SentenceIdPair> sentpairList, int topN)
        {
            SortedDictionary<int, List<SentenceIdPair>> sorted = new SortedDictionary<int, List<SentenceIdPair>>();
            List<SentencePair> r = new List<SentencePair>();

            foreach (SentenceIdPair item in sentpairList)
            {
                if (sorted.ContainsKey(item.score) == false)
                {
                    sorted.Add(item.score, new List<SentenceIdPair>());
                }
                sorted[item.score].Add(item);
            }

            HashSet<int> savedOffset = new HashSet<int>();
            foreach (KeyValuePair<int, List<SentenceIdPair>> pair in sorted)
            {
                foreach (SentenceIdPair item in pair.Value)
                {
                    if (savedOffset.Contains(item.sentPairOffset) == true)
                    {
                        continue;
                    }
                    savedOffset.Add(item.sentPairOffset);

                   
                    if (topN <= 0)
                    {
                        return r;
                    }

                    SentencePair sp = new SentencePair();

                    sentPairReader.Read(out sp.Language1Line, out sp.Language2Line, out sp.strUrl, item.sentPairOffset);
                    sp.score = item.score;

                    r.Add(sp);
                    topN--;
                }
            }

            return r;
        }

        /// <summary>
        /// 求一个字符串的回溯函数。
        /// 约定序列下标从0开始。
        /// 回溯函数是整数集[0,n-1]到N的映射，n为字符串的长度。
        /// 回溯函数的定义：
        /// 设存在非空序列L，i为其合法下标；
        /// L[i]的前前置序列集为：{空集,L中所有以i-1为最后一个元素下标的子序列}
        /// L的前置序列集为：{空集,L中所有以0为第一个元素下标的子序列}
        /// 下标i的回溯函数值的定义为：
        /// 如果i=0,回溯函数值为-1
        /// 否则i的回溯函数值为i的前置序列集和L的前置序列集中相等元素的最大长度
        /// 换句话说是，设集合V={x,x属于i的前置序列集,并且x属于L的前置序列集，并且x的长度小于i}，回溯函数值为max(V)
        /// 当i=0时并不存在这样的一个x，所以约定此时的回溯函数值为-1
        /// 回溯函数的意义：
        /// 如果标号为j的字符同主串失配，那么将子串回溯到next[j]继续与主串匹配，如果next[j]=-1,则主串的匹配点后移一位，同子串的第一个元素开始匹配。
        /// 同一般的模式匹配算法相比，kmp通过回溯函数在失配的情况下跳过了若干轮匹配(向右滑动距离可能大于1)
        /// kmp算法保证跳过去的这些轮匹配一定是失配的，这一点可以证明
        /// </summary>
        /// <param name="pattern">模式串，上面的注释里将其称为子串</param>
        /// <returns>回溯函数是kmp算法的核心，本函数依照其定义求出回溯函数，KMP函数依照其意义使用回溯函数。</returns>
        public static int[] Next(ref int [] pattern)
        {
            int[] next = new int[pattern.Length];
            next[0] = -1;
            if (pattern.Length < 2) //如果只有1个元素不用kmp效率会好一些
            {
                return next;
            }

            next[1] = 0;    //第二个元素的回溯函数值必然是0，可以证明：
            //1的前置序列集为{空集,L[0]}，L[0]的长度不小于1，所以淘汰，空集的长度为0，故回溯函数值为0
            int i = 2;  //正被计算next值的字符的索引
            int j = 0;  //计算next值所需要的中间变量，每一轮迭代初始时j总为next[i-1]
            while (i < pattern.Length)    //很明显当i==pattern.Length时所有字符的next值都已计算完毕，任务已经完成
            { //状态点
                if (pattern[i - 1] == pattern[j])   //首先必须记住在本函数实现中，迭代计算next值是从第三个元素开始的
                {   //如果L[i-1]等于L[j]，那么next[i] = j + 1
                    next[i++] = ++j;
                }
                else
                {   //如果不相等则检查next[i]的下一个可能值----next[j]
                    j = next[j];
                    if (j == -1)    //如果j == -1则表示next[i]的值是1
                    {   //可以把这一部分提取出来与外层判断合并
                        //书上的kmp代码很难理解的一个原因就是已经被优化，从而遮蔽了其实际逻辑
                        next[i++] = ++j;
                    }
                }
            }
            return next;
        }


        /// <summary>
        /// KMP函数同普通的模式匹配函数的差别在于使用了next函数来使模式串一次向右滑动多位称为可能
        /// next函数的本质是提取重复的计算
        /// </summary>
        /// <param name="source">主串</param>
        /// <param name="pattern">用于查找主串中一个位置的模式串</param>
        /// <returns>-1表示没有匹配，否则返回匹配的标号</returns>
        public static int ExecuteKMP(ref List<int> source, ref int [] pattern)
        {
            int[] next = Next(ref pattern);
            int i = 0;  //主串指针
            int j = 0;  //模式串指针
            //如果子串没有匹配完毕并且主串没有搜索完成
            while (j < pattern.Length && i < source.Count)
            {
                if (source[i] == pattern[j])    //i和j的逻辑意义体现于此，用于指示本轮迭代中要判断是否相等的主串字符和模式串字符
                {
                    i++;
                    j++;
                }
                else
                {
                    j = next[j];    //依照指示迭代回溯
                    if (j == -1)    //回溯有情况，这是第二种
                    {
                        i++;
                        j++;
                    }
                }
            }
            //如果j==pattern.Length则表示循环的退出是由于子串已经匹配完毕而不是主串用尽
            return j < pattern.Length ? -1 : i - j;
        }


        private void CalcLanguage1SentencePairRelevance(ref SortedList<int, QueryTermProp> queryIdList, ref List<SentenceIdPair> sentpairList, ref List<int> orgQueryTermIdList)
        {
            List<SentenceIdPair> resultList = new List<SentenceIdPair>();

            int j;
            for (j = 0; j < sentpairList.Count; j++)
            {
                int r = 0;
                bool bFlag = true;

                if (orgQueryTermIdList.Count == 0 || ExecuteKMP(ref orgQueryTermIdList, ref sentpairList[j].aiLanguage1Line) == -1)
                {
                    bFlag = CalcInvertedSequence(ref queryIdList, orgQueryTermIdList.Count, ref sentpairList[j].aiLanguage1Line, out r);
                }

                r += sentpairList[j].aiLanguage1Line.Length;
                sentpairList[j].score = r;
            }
        }


        private void CalcSentencePairRelevance(ref SortedList<int, QueryTermProp> queryIdList, ref List<SentenceIdPair> sentpairList, ref List<int> orgQueryTermIdList)
        {
            List<SentenceIdPair> resultList = new List<SentenceIdPair>();

            int j;
            for (j = 0; j < sentpairList.Count; j++)
            {
                int r = 0;
                bool bFlag = true;

                if (orgQueryTermIdList.Count == 0 || ExecuteKMP(ref orgQueryTermIdList, ref sentpairList[j].aiLanguage2Line) == -1)
                {
                    bFlag = CalcInvertedSequence(ref queryIdList, orgQueryTermIdList.Count, ref sentpairList[j].aiLanguage2Line, out r);
                }

                r += sentpairList[j].aiLanguage2Line.Length;
                sentpairList[j].score = r;
            }
        }

        private bool IsPunctuation(int termId)
        {
            string str;
            str = dictReader.GetStrTermById(termId);
            if (str == "。" ||
                str == "，" ||
                str == "," ||
                str == "!" ||
                str == "+" ||
                str == "." ||
                str == "?" ||
                str == "、" ||
                str == ":" ||
                str == ";" ||
                str == "·" ||
                str == "…")
            {
                return true;
            }
            else
            {
                return false;
            }
        }

        private bool CalcInvertedSequence(ref SortedList<int, QueryTermProp> queryIdList, int queryLength, ref int [] calcWordList, out int score)
        {
            List<int> IdList = new List<int>();
            bool bFirst = true;
            int totalTermSpace = 0;
            int lastTermIndex = 0;

            int goodSynScore = 0;
            int normalSynScore = 0;
            int badSynScore = 0;

            int tempPunctuationNum = 0;
            int punctuationNum = 0;
            int i = 0;

            int[] termOccupyBitmap;
            termOccupyBitmap = new int[queryLength];
            for (i = 0; i < termOccupyBitmap.Length; i++)
            {
                termOccupyBitmap[i] = 0;
            }

            i = 0;
            foreach (int item in calcWordList)
            {
                if (bFirst == false && IsPunctuation(item) == true)
                {
                    tempPunctuationNum++;
                }

                int id;
                QueryTermProp qtProp;
                if (queryIdList.TryGetValue(item, out qtProp) == true)
                {
                    termOccupyBitmap[qtProp.offset]++;
                    switch (qtProp.synType)
                    {
                        case 0:
                            break;
                        case 1:
                        case 7:
                            goodSynScore++;
                            break;
                        case 2:
                        case 5:
                            normalSynScore++;
                            break;
                        case 6:
                        case 9:
                            badSynScore++;
                            break;
                        default:
                            break;
                    }
                    IdList.Add(qtProp.offset);

                    if (bFirst == false)
                    {
                        totalTermSpace += (i - lastTermIndex - 1);
                        punctuationNum += tempPunctuationNum;
                        tempPunctuationNum = 0;
                    }
                    else
                    {
                        bFirst = false;
                    }
                    lastTermIndex = i;
                }
                i++;
            }

            int r = 0;
            RecurCalcIS(ref IdList, 0, IdList.Count, ref r);

            int finePoint = 0;
            for (i = 0; i < termOccupyBitmap.Length; i++)
            {
                if (termOccupyBitmap[i] == 0)
                {
                    finePoint++;
                }
            }

            score = (r * 100 + totalTermSpace + badSynScore * 10 + normalSynScore * 5 + punctuationNum * 50 + finePoint * 150) * iRankLevelFactor + goodSynScore * 2;

            if (finePoint == 0)
            {
                return true;
            }

            return false;
        }

        private void RecurCalcIS(ref List<int> IdList, int l, int r, ref int cnt)
        {
            int mid, i, j, tmp;
            int[] c;

            c = new int[IdList.Count];

            if (r > l + 1)
            {
                mid = (l + r) / 2;
                RecurCalcIS(ref IdList, l, mid, ref cnt);
                RecurCalcIS(ref IdList, mid, r, ref cnt);
                tmp = l;

                for (i = l, j = mid; i < mid && j < r; )
                {

                    if (IdList[i] > IdList[j])
                    {
                        c[tmp++] = IdList[j++];
                        cnt += mid - i;
                    }
                    else
                    {
                        c[tmp++] = IdList[i++];
                    }
                }
                if (j < r)
                {
                    for (; j < r; ++j)
                    {
                        c[tmp++] = IdList[j];
                    }
                }
                else
                {
                    for (; i < mid; ++i)
                    {
                        c[tmp++] = IdList[i];
                    }
                }

                for (i = l; i < r; ++i)
                {
                    IdList[i] = c[i];
                }
            }
        }

        public List<int> ConvertStrTermListToTermIdList(ref List<string> strTermList)
        {
            List<int> termIdList = new List<int>();
            foreach (string item in strTermList)
            {
                int termId;
                termId = dictReader.GetTermId(item);
             //   if (termId != -1)
            //    {
                    termIdList.Add(termId);
            //    }
            }

            return termIdList;
        }


        //日文例句搜索
        public List<SentencePair> QuerySentencePairByJapanese(string strQuery, int topN)
        {
            List<string> wordList = jpnWordBreaker.SplitSentence(strQuery, true);
            List<int> synWordList;
            List<int> queryTermIdList = new List<int>();
            SortedList<int, QueryTermProp> id2prop = new SortedList<int, QueryTermProp>();


            //关键词被切分为多个词, 不进行同义词扩展

            int pos = 0;
            foreach (string item in wordList)
            {
                int termId;
                termId = dictReader.GetTermId(item);
                queryTermIdList.Add(termId);
                List<string> strTermList = jpnWordBreaker.getSynonmy(item);
                synWordList = ConvertStrTermListToTermIdList(ref strTermList);
                foreach (int synId in synWordList)
                {
                    QueryTermProp prop = new QueryTermProp();
                    prop.offset = pos;
                    if (synId != termId)
                    {
                        prop.synType = 2;
                    }
                    else
                    {
                        prop.synType = 0;
                    }
                    prop.termId = synId;
                    prop.synSrcTerm = termId;
                    if (id2prop.ContainsKey(synId) == false)
                    {
                        id2prop.Add(synId, prop);
                    }
                }
                pos++;
            }

            List<SentenceIdPair> sentpairList = null;

            Dictionary<int, int[]> word2SentId = QueryWord2SentencePairId(ref id2prop, ref language1TermIndexReader);
            List<int> commonSentenceIndex = QueryCommonSentence(ref word2SentId);
            if (commonSentenceIndex != null)
            {
                if (commonSentenceIndex.Count > 0)
                {
                    sentpairList = GetSentencePairByIDList(ref commonSentenceIndex);
                }
            }

            if (sentpairList != null && sentpairList.Count > 0)
            {
                CalcLanguage1SentencePairRelevance(ref id2prop, ref sentpairList, ref queryTermIdList);
                return RankSentencePair(ref sentpairList, topN);
            }

            return new List<SentencePair>();
        }

        public List<SentencePair> QuerySentencePairByChinese(string strQuery, int topN)
        {
            List<string> wordList = chsWordBreaker.ParseString(ref strQuery, 30, true);
            List<int> queryTermIdList = new List<int>();
            SortedList<int, QueryTermProp> queryIdList = new SortedList<int, QueryTermProp>();

            int pos = 0;
            foreach (string item in wordList)
            {
                int termId;
                termId = dictReader.GetTermId(item);
                queryTermIdList.Add(termId);

                Dictionary<string, int> d = chsSynonym.GetSynonymType(item);
                foreach (KeyValuePair<string, int> pair in d)
                {
                    int synTermId;
                    synTermId = dictReader.GetTermId(pair.Key);
                    if (synTermId >= 0)
                    {
                        if (queryIdList.ContainsKey(synTermId) == false)
                        {
                            QueryTermProp prop = new QueryTermProp();
                            prop.offset = pos;
                            prop.synSrcTerm = termId;
                            prop.synType = pair.Value;
                            prop.termId = synTermId;

                            queryIdList.Add(synTermId, prop);

                        }
                    }
                }
                pos++;
            }

            List<SentenceIdPair> sentpairList = null;
            Dictionary<int, int[]> word2SentId = QueryWord2SentencePairId(ref queryIdList, ref language2TermIndexReader);
            List<int> commonSentenceIndex = QueryCommonSentence(ref word2SentId);
            if (commonSentenceIndex != null)
            {
                if (commonSentenceIndex.Count > 0)
                {
                    sentpairList = GetSentencePairByIDList(ref commonSentenceIndex);
                }
            }

            if (sentpairList != null && sentpairList.Count > 0)
            {
                CalcSentencePairRelevance(ref queryIdList, ref sentpairList, ref queryTermIdList);
                return RankSentencePair(ref sentpairList, topN);
            }

            return new List<SentencePair>();
        }


        public List<SentencePair> QuerySentencePairByEnglish(string word, int topN)
        {
            string[] wordList;
            char[] sep = { ' '};

            int queryTermId;
            queryTermId = dictReader.GetTermId(word);

            wordList = word.ToLower().Trim().Split(sep);
            List<string> termList = new List<string>(wordList);
            List<int> termIdList;
            termIdList = ConvertStrTermListToTermIdList(ref termList);

            
            Dictionary<int, int[]> word2SentId = new Dictionary<int, int[]>();
            foreach (int termId in termIdList)
            {
                if (word2SentId.ContainsKey(termId) == false)
                {
                    int[] aiIndex = language1TermIndexReader.ReadTermSentenceIndex(termId);
                    if (aiIndex == null)
                    {
                        return new List<SentencePair>();
                    }
                    word2SentId.Add(termId, aiIndex);
                }
            }

            List<SentenceIdPair> sentpairList = null;
            List<int> commonSentenceIndex = null;
            if (word2SentId.Count > 0)
            {
                commonSentenceIndex = QueryCommonSentence(ref word2SentId);
                if (commonSentenceIndex == null)
                {
                    return new List<SentencePair>();
                }
            }

            if (commonSentenceIndex.Count > 0)
            {
                sentpairList = GetSentencePairByIDList(ref commonSentenceIndex);
            }

            if (sentpairList != null && sentpairList.Count > 0)
            {
                SortedList<int, QueryTermProp> queryIdList = new SortedList<int, QueryTermProp>();
                int pos = 0;
                foreach (int termId in termIdList)
                {
                    if (queryIdList.ContainsKey(termId) == false)
                    {
                        QueryTermProp prop = new QueryTermProp();
                        prop.offset = pos;
                        prop.synSrcTerm = termId;
                        prop.synType = 0;
                        prop.termId = termId;

                        queryIdList.Add(termId, prop);

                        pos++;
                    }
                }

                int i;
                for (i = 0;i < sentpairList.Count;i++)
                {
                    int r = 0;

                    if (termIdList.Count == 0 || ExecuteKMP(ref termIdList, ref sentpairList[i].aiLanguage1Line) == -1)
                    {
                        CalcInvertedSequence(ref queryIdList, queryIdList.Count, ref sentpairList[i].aiLanguage1Line, out r);
                    }
                    r += sentpairList[i].aiLanguage1Line.Length;

                    sentpairList[i].score = r;
                }
                return RankSentencePair(ref sentpairList, topN);
            }
            return new List<SentencePair>();
            
        }

        #endregion
    }
}
